Data Science Project - Spotify

by Bruno Miguel Pereira Gomes, Lorenzo Adam Piazza

• https://developer.spotify.com/documentation/web-api/reference/get-audio-features

# Load datasets

import json
import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
import plotly.subplots as ps

df_songs = pd.read_csv('songs_extended.csv', index_col = 'id')
df_songs['release_date'] = pd.to_datetime(df_songs['release_date'], format = '%Y-%m-%d', errors = 'coerce')

df_genres = pd.read_csv('genres.csv', index_col = 'genre')

df_artists = pd.read_csv('artists.csv', index_col = 'id')
df_artists['first_song'] = pd.to_datetime(df_artists['first_song'], format = '%Y-%m-%d', errors = 'coerce')

unnormalized_history = pd.read_json('my_spotify_data/Spotify Extended Streaming History/Streaming_History_Audio_2018-2023.json', typ='series')
df_allHistory = pd.json_normalize(unnormalized_history["songs"])

df_songs.info()

<class 'pandas.core.frame.DataFrame'>
Index: 1204025 entries, 7lmeHLHBe4nmXzuXc0HDjk to 3GgQmOxxLyRoAb4j86zOBX
Data columns (total 26 columns):
 #   Column             Non-Null Count    Dtype         
---  ------             --------------    -----         
 0   name               1204025 non-null  object        
 1   album              1204025 non-null  object        
 2   album_id           1204025 non-null  object        
 3   artists            1204025 non-null  object        
 4   artist_ids         1204025 non-null  object        
 5   track_number       1204025 non-null  int64         
 6   disc_number        1204025 non-null  int64         
 7   explicit           1204025 non-null  bool          
 8   danceability       1204025 non-null  float64       
 9   energy             1204025 non-null  float64       
 10  key                1204025 non-null  int64         
 11  loudness           1204025 non-null  float64       
 12  mode               1204025 non-null  int64         
 13  speechiness        1204025 non-null  float64       
 14  acousticness       1204025 non-null  float64       
 15  instrumentalness   1204025 non-null  float64       
 16  liveness           1204025 non-null  float64       
 17  valence            1204025 non-null  float64       
 18  tempo              1204025 non-null  float64       
 19  duration_ms        1204025 non-null  int64         
 20  time_signature     1204025 non-null  float64       
 21  year               1204025 non-null  int64         
 22  release_date       1204015 non-null  datetime64[ns]
 23  genres             1204025 non-null  object        
 24  categories         1204025 non-null  object        
 25  artist_popularity  1204025 non-null  int64         
dtypes: bool(1), datetime64[ns](1), float64(10), int64(7), object(7)
memory usage: 240.0+ MB

df_songs.head()

	name	album	album_id	artists	artist_ids	track_number	disc_number	explicit	danceability	energy	...	liveness	valence	tempo	duration_ms	time_signature	year	release_date	genres	categories	artist_popularity
id
7lmeHLHBe4nmXzuXc0HDjk	Testify	The Battle Of Los Angeles	2eia0myWFgoHuttJytCxgX	['Rage Against The Machine']	['2d0hyoQ5ynDBnkvAbJKORj']	1	1	False	0.470	0.978	...	0.3560	0.503	117.906	210133	4.0	1999	1999-11-02	['alternative metal', 'alternative rock', 'con...	['hiphop', 'metal', 'other', 'rap', 'rock']	68
1wsRitfRRtWyEapl0q22o8	Guerrilla Radio	The Battle Of Los Angeles	2eia0myWFgoHuttJytCxgX	['Rage Against The Machine']	['2d0hyoQ5ynDBnkvAbJKORj']	2	1	True	0.599	0.957	...	0.1550	0.489	103.680	206200	4.0	1999	1999-11-02	['alternative metal', 'alternative rock', 'con...	['hiphop', 'metal', 'other', 'rap', 'rock']	68
1hR0fIFK2qRG3f3RF70pb7	Calm Like a Bomb	The Battle Of Los Angeles	2eia0myWFgoHuttJytCxgX	['Rage Against The Machine']	['2d0hyoQ5ynDBnkvAbJKORj']	3	1	False	0.315	0.970	...	0.1220	0.370	149.749	298893	4.0	1999	1999-11-02	['alternative metal', 'alternative rock', 'con...	['hiphop', 'metal', 'other', 'rap', 'rock']	68
2lbASgTSoDO7MTuLAXlTW0	Mic Check	The Battle Of Los Angeles	2eia0myWFgoHuttJytCxgX	['Rage Against The Machine']	['2d0hyoQ5ynDBnkvAbJKORj']	4	1	True	0.440	0.967	...	0.1210	0.574	96.752	213640	4.0	1999	1999-11-02	['alternative metal', 'alternative rock', 'con...	['hiphop', 'metal', 'other', 'rap', 'rock']	68
1MQTmpYOZ6fcMQc56Hdo7T	Sleep Now In the Fire	The Battle Of Los Angeles	2eia0myWFgoHuttJytCxgX	['Rage Against The Machine']	['2d0hyoQ5ynDBnkvAbJKORj']	5	1	False	0.426	0.929	...	0.0789	0.539	127.059	205600	4.0	1999	1999-11-02	['alternative metal', 'alternative rock', 'con...	['hiphop', 'metal', 'other', 'rap', 'rock']	68

5 rows × 26 columns

df_genres.info()

<class 'pandas.core.frame.DataFrame'>
Index: 5265 entries, 2-step to zydeco
Data columns (total 1 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   category  5265 non-null   object
dtypes: object(1)
memory usage: 82.3+ KB

df_genres.head()

	category
genre
2-step	other
21st century classical	classic
432hz	other
48g	other
5th wave emo	other

df_artists.info()

<class 'pandas.core.frame.DataFrame'>
Index: 140064 entries, 7F4HcalxCMC4DctguvnoFY to 1tsNLehJWv67iMipy0WwQR
Data columns (total 6 columns):
 #   Column      Non-Null Count   Dtype         
---  ------      --------------   -----         
 0   name        140061 non-null  object        
 1   genres      140064 non-null  object        
 2   category    140064 non-null  object        
 3   followers   140064 non-null  int64         
 4   popularity  140064 non-null  int64         
 5   first_song  139540 non-null  datetime64[ns]
dtypes: datetime64[ns](1), int64(2), object(3)
memory usage: 7.5+ MB

df_artists.head()

	name	genres	category	followers	popularity	first_song
id
7F4HcalxCMC4DctguvnoFY	Ace Wilder	['melodipop', 'swedish pop']	['pop']	29030	23	2014-02-23
21QpKYI2IXZRv21rLrbNZy	Rolf Gjelsten	[]	[]	2	0	2015-02-03
1gQ1kamk2cHpy59Zyd1TPp	Sophia McKenna	[]	[]	9	15	1989-11-01
5XDrmlSI7dXpJkiPb3Oqd4	Underdog	['hardcore', 'new jersey hardcore', 'nyhc']	['other']	10586	14	1996-01-01
58wjOQFamj752bO2Bbjjug	Veer	['dubstep', 'indian electronic', 'riddim dubst...	['electronic', 'other']	6198	10	2020-01-24

# Distribution of song durations (0 - 10 minutes)

df_songs['duration_sec'] = df_songs['duration_ms'] // 1000
df_songs['duration_min'] = df_songs['duration_sec'] / 60
df_songs['is_over_10_mins'] = df_songs['duration_min'] > 10

num_of_songs = len(df_songs)
num_of_songs_below_10_minutes = num_of_songs - df_songs["is_over_10_mins"].sum()
num_of_songs_above_10_minutes = df_songs["is_over_10_mins"].sum()

print(f'Number of songs below 10 minutes: { num_of_songs_below_10_minutes } ( { num_of_songs_below_10_minutes / num_of_songs * 100 }% )')
print(f'Number of songs over 10 minutes: { num_of_songs_above_10_minutes } ( { num_of_songs_above_10_minutes / num_of_songs * 100 }% )')
print(f'Average song duration: {df_songs["duration_min"].mean():.0f} minutes')

px.histogram(data_frame = df_songs, x = 'duration_min', range_x = [0, 10], nbins = 500) \
    .update_layout(title = 'Distribution of song durations (0 - 10 minutes)', xaxis_title = 'Song duration (Minutes)', yaxis_title = 'Number of Songs')

Number of songs below 10 minutes: 1176239 ( 97.69224060962189% )
Number of songs over 10 minutes: 27786 ( 2.3077593903781066% )
Average song duration: 4 minutes

# Song releases per year

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year).size()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Song releases per year', xaxis_title = 'Release date', yaxis_title = 'Number of Songs', showlegend = False)

# New artists through time

df_tmp = df_artists.groupby(df_artists['first_song'].dt.year)['name'].count()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'New artists', xaxis_title = 'Release date', yaxis_title = 'Number of new artists', showlegend = False)

# Percentage of songs of genre 'Disco'

df_songs['is_disco'] = df_songs['categories'].str.contains('disco')

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_disco'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'Disco\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'Disco\' (%)', showlegend = False, yaxis_ticksuffix = "%") \
    .add_vline(x = 1960, line_color = 'red') \
    .add_vline(x = 1980, line_color = 'red')

# Danceability through time

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year)['danceability'].mean()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Danceability through time', xaxis_title = 'Release date', yaxis_title = 'Average danceability', showlegend = False) \
    .add_vline(x = 1960, line_color = 'red') \
    .add_vline(x = 1980, line_color = 'red')

# Acousticness through time

# https://www.softschools.com/timelines/music_genres_timeline/428/
# https://en.wikipedia.org/wiki/1980s_in_music
# 'The 1980s saw the emergence of electronic dance music and new wave, also known as
#   Modern Rock.'
# The 1980s are commonly remembered for a great increase in the use of digital recording,
#   associated with the usage of synthesizers, with synth-pop music and other electronic
#   genres featuring non-traditional instruments increasing in popularity.

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year)['acousticness'].mean()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Acousticness through time', xaxis_title = 'Release date', yaxis_title = 'Average acousticness', showlegend = False) \
    .add_vline(x = 1960, line_color = 'red') \
    .add_vline(x = 1980, line_color = 'red')

# Energy through time

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year)['energy'].mean()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Energy through time', xaxis_title = 'Release date', yaxis_title = 'Average energy', showlegend = False) \
    .add_vline(x = 1960, line_color = 'red') \
    .add_vline(x = 1980, line_color = 'red')

# Tempo through time

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year)['tempo'].mean()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Tempo through time', xaxis_title = 'Release date', yaxis_title = 'Average tempo', showlegend = False) \
    .add_vline(x = 1960, line_color = 'red') \
    .add_vline(x = 1980, line_color = 'red')

# Loudness through time

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year)['loudness'].mean()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Loudness through time', xaxis_title = 'Release date', yaxis_title = 'Average loudness', showlegend = False) \
    .add_vline(x = 1960, line_color = 'red') \
    .add_vline(x = 1980, line_color = 'red')

# Percentage of songs of genre 'Metal'

df_songs['is_metal'] = df_songs['categories'].str.contains('metal')

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_metal'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'Metal\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'Metal\' (%)', showlegend = False, yaxis_ticksuffix = "%")

# Percentage of songs of genre 'Rock'

df_songs['is_rock'] = df_songs['categories'].str.contains('rock')

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_rock'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'Rock\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'Rock\' (%)', showlegend = False, yaxis_ticksuffix = "%")

# Percentage of songs of genre 'Rap'

df_songs['is_rap'] = df_songs['categories'].str.contains('rap')

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_rap'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'Rap\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'Rap\' (%)', showlegend = False, yaxis_ticksuffix = "%")

# Percentage of songs of genre 'Punk'

df_songs['is_punk'] = df_songs['categories'].str.contains('punk')

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_punk'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'Punk\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'Punk\' (%)', showlegend = False, yaxis_ticksuffix = "%")

# Percentage of songs of genre 'K-Pop'

df_songs['is_kpop'] = df_songs['categories'].str.contains('k-pop')

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_kpop'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'K-Pop\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'K-Pop\' (%)', showlegend = False, yaxis_ticksuffix = "%")

# Percentage of songs of genre 'Pop'

df_songs['is_pop'] = df_songs['categories'].str.contains("'pop'")

df_tmp = (df_songs.groupby(df_songs['release_date'].dt.year)['is_pop'].sum() / df_songs.groupby(df_songs['release_date'].dt.year)['name'].count()) * 100.0

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Percentage of songs of genre \'Pop\'', xaxis_title = 'Release date', yaxis_title = 'Songs of genre \'Pop\' (%)', showlegend = False, yaxis_ticksuffix = "%")

# Number of explicit songs through Time

df_tmp = df_songs.groupby(df_songs['release_date'].dt.year)['explicit'].sum()

px.line(data_frame = df_tmp) \
    .update_layout(title = 'Number of explicit songs through Time', xaxis_title = 'Release date', yaxis_title = 'Number of songs', showlegend = False)

# The popularity of the artist. The value will be between 0 and 100, with 100 being the most popular.
# The artist's popularity is calculated from the popularity of all the artist's tracks.

df_tmp = df_artists.sort_values(by = 'popularity', ascending = False)

px.bar(data_frame = df_tmp[0:10], x = 'popularity', y = 'name') \
    .update_layout(title = 'Top 10 most popular artists (as of November 2023)', xaxis_title = 'Popularity', yaxis_title = 'Artist name') \
    .update_yaxes(categoryorder = 'total ascending')

# Distribution of popularity

df_tmp = df_artists.groupby(df_artists['popularity'])['name'].count()

px.bar(data_frame = df_tmp) \
    .update_layout(title = 'Distribution of popularity', xaxis_title = 'Popularity', yaxis_title = 'Number of artists', showlegend = False)

# Top 10 most popular genres (limited to the top 1000 artists)

top_genres = []

df_tmp = df_artists.sort_values(by = 'popularity', ascending = False)

for index, artist in df_tmp[:1000].iterrows():
    if artist['category'] == '[]':
        continue
    
    artist_genres = artist['category'][2:-2].split("', '")
    
    top_genres.extend(artist_genres)

top_genres = np.unique(top_genres, return_counts = True)

df_tmp = pd.DataFrame({
    'genre': pd.Series(top_genres[0]),
    'popularity': pd.Series(top_genres[1])
})

df_tmp = df_tmp.sort_values(by = 'popularity', ascending = False)

px.bar(data_frame = df_tmp[:10], x = 'popularity', y = 'genre') \
    .update_layout(title = 'Top 10 most popular genres (limited to the top 1000 artists)', xaxis_title = 'Number of songs', yaxis_title = 'Genre') \
    .update_yaxes(categoryorder = 'total ascending')

# Distribution of variations of genres

df_tmp = df_genres
df_tmp['genre'] = df_tmp.index

df_tmp = df_tmp.groupby(df_tmp['category'], as_index = False).count()

px.pie(data_frame = df_tmp, values = 'genre', names = 'category') \
    .update_layout(title = 'Distribution of variations of genres')

Bruno's Dataset Section

counting = df_allHistory.groupby(df_allHistory["reason_start"]).count()
counting = counting.reset_index()

masking = (counting['reason_start'].eq('trackdone')) | (counting['reason_start'].eq('clickrow')) | (counting['reason_start'].eq('playbtn'))

px.pie(counting[masking], values="ts", names="reason_start") \
            .update_traces(textinfo="percent+label") \
            .update_layout(title="Reasons a track started", height=700)

counting = df_allHistory.groupby(df_allHistory["reason_end"]).count()
counting = counting.reset_index()

masking = (counting['reason_end'].eq('trackdone')) | (counting['reason_end'].eq('endplay')) | (counting['reason_end'].eq('fwdbtn'))

px.pie(counting[masking], values="ts", names="reason_end") \
            .update_traces(textinfo="percent+label") \
            .update_layout(title="Reasons a track ended", height=700)

df_allHistory["ts"] = pd.to_datetime(df_allHistory["ts"])


fig = px.histogram(df_allHistory, x="ts", nbins=65) \
            .update_layout(title="Number of listens per month throughout my account's lifespan", xaxis_title="Time", yaxis_title="Number of listens")

years = ["2018", "2019", "2020", "2021", "2022", "2023", "2024"]
for i in years:
    fig.add_shape(
        type="line",
        x0 = i + "-01-01 00:00:00+00:00",
        y0 = 0,
        x1 = i + "-01-01 00:00:00+00:00",
        y1 = 600,
        line=dict(
            color = "black",
            width = 1,
            dash = "dot"
        )
    )

fig.show()

def true_platform(platform):
    if "ios" in platform.lower():
        return 'iOS'
    elif "web" in platform.lower():
        return 'web'
    elif "windows" in platform.lower():
        return 'windows'
    else:
        return 'error'


tabellaPlatforms = df_allHistory

tabellaPlatforms["truePlatform"] = tabellaPlatforms['platform'].apply(true_platform)

px.pie(tabellaPlatforms, names="truePlatform") \
            .update_traces(textinfo="percent+label") \
            .update_layout(title="Number of listens on different devices", height=700)

counting = df_allHistory.groupby(['master_metadata_track_name', 'master_metadata_album_artist_name']).count()


masking = counting >= 25


trackFromYear = counting[masking]["ts"].dropna()
trackFromYear = trackFromYear.reset_index()


px.histogram(trackFromYear, y="master_metadata_track_name", x="ts", color="master_metadata_album_artist_name") \
            .update_yaxes(categoryorder="total ascending") \
            .update_layout(title="Number of listens of a certain song (Minimum 25 listens)", \
                           yaxis_title="Song", xaxis_title="Number of listens", height=700)

counting = df_allHistory.groupby(df_allHistory["master_metadata_album_artist_name"]).count()

artistsFromYear = counting["ts"].dropna()
artistsFromYear = artistsFromYear.reset_index()

df_sorted = artistsFromYear.sort_values('ts', ascending=False)

top5ArtistiAscoltati = df_allHistory.loc[df_allHistory['master_metadata_album_artist_name'].isin(df_sorted.head(5)["master_metadata_album_artist_name"])]
altriArtistiAscoltati = df_allHistory.loc[~df_allHistory['master_metadata_album_artist_name'].isin(df_sorted.head(5)["master_metadata_album_artist_name"])]

provaDebole = top5ArtistiAscoltati.groupby(top5ArtistiAscoltati["master_metadata_album_artist_name"]).count()
provaDebole = provaDebole.reset_index()

superProva = altriArtistiAscoltati.groupby(altriArtistiAscoltati["master_metadata_album_artist_name"]).count()

maskingLower = superProva <= 5
maskingHigher = superProva > 5

aaa = counting[maskingLower]["ts"].dropna()
bbb = counting[maskingHigher]["ts"].dropna()

aaa = aaa.reset_index()
bbb = bbb.reset_index()

xxx = aaa["ts"].sum()
yyy = bbb["ts"].sum()

new_df = provaDebole.loc[:, ['master_metadata_album_artist_name', 'ts']]

new_row1 = {'master_metadata_album_artist_name':'Over 5 Songs Listened', 'ts':xxx}
new_row2 = {'master_metadata_album_artist_name':'Other Artists', 'ts':yyy}

new_df.loc[len(new_df)] = new_row1
new_df.loc[len(new_df)] = new_row2

color_map = { "Tyler, The Creator": "rgb(13, 8, 135)",
            "Childish Gambino": "rgb(126, 3, 168)",
            "Frank Ocean": "rgb(204, 71, 120)",
            "coldrain": "rgb(248, 149, 64)",
            "Kendrick Lamar": "rgb(240, 249, 33)",
            "Over 5 Songs Listened": "darkgrey",
            "Other Artists": "grey"}

px.pie(new_df, values="ts", names="master_metadata_album_artist_name", color="master_metadata_album_artist_name", color_discrete_map=color_map) \
            .update_traces(textinfo="percent+label+value") \
            .update_layout(title="Disparity between my most listened to artists vs. others", height=700)

#BE CAREFUL EXECUTING THIS ONE, IT TAKES A WHILE (51 seconds ~)
pd.options.mode.chained_assignment = None
df_artists = pd.read_csv('artists.csv') #Have to reset it, because we have different algorythms for df_artists

df_allHistory["genres"] = ""

for index, row in df_allHistory.iterrows():
    nameMask = df_artists["name"] == row["master_metadata_album_artist_name"]
    df_allHistory["genres"][index] = df_artists[nameMask]["category"].to_string(index=False)
    #print(row["master_metadata_album_artist_name"])
    #print(df_allHistory["genres"][index])
    #print("---------------------------------------------")



print("finished")

finished

# Most popular genres

top_genres = []

for index, row in df_allHistory.iterrows():
    if not row['genres'].startswith("['"):
        continue
    if not row['genres'].endswith("']"):
        continue
    artist_genres = row['genres'][2:-2].split("', '")
    top_genres.extend(artist_genres)


top_genres = np.unique(top_genres, return_counts = True)

generiAscoltati = pd.DataFrame({
    'genre': pd.Series(top_genres[0]),
    'popularity': pd.Series(top_genres[1])
})

generiAscoltati = generiAscoltati.sort_values(by = 'popularity', ascending = False)

px.histogram(generiAscoltati, x = "popularity", y="genre") \
            .update_yaxes(categoryorder="total ascending") \
            .update_layout(title="Preferred genres", xaxis_title="Number of listens", yaxis_title="Genre", height=500)

Dataset generators

def sanitize(string: str) -> str: return string \ .replace('\'', '_') \ .replace('"', '_') \ .replace('[', '_') \ .replace(']', '_') \ .replace(',','_') def sanitize_list(list_: list) -> list: sanitized_list = [] for element in list_: sanitized_list.append( sanitize(element) ) return sanitized_list # Gather all the unique artist IDs and dump them to a file. import pandas as pd df_songs = pd.read_csv('data/tracks_features.csv', index_col = 'id') artist_ids = [] for artists_string in df_songs['artist_ids']: artists = artists_string[2:-2].split("', '") for artist_id in artists: artist_ids.append(artist_id) artist_ids = np.unique(artist_ids) with open('artist_ids.txt', 'w') as out_file: for artist in artist_ids: out_file.write(artist + '\n') # Use the Spotify API to download information about the artists. # Requires Spotify Developer tokens import json import time import spotipy CID = '' SECRET = '' client_credentials_manager = spotipy.oauth2.SpotifyClientCredentials(client_id = CID, client_secret = SECRET) sp = spotipy.Spotify(client_credentials_manager = client_credentials_manager) with open('artist_ids.txt', 'r') as fin: lines = fin.readlines() lines_n = len(lines) batch_artists = [] for index, line in enumerate(lines): artist_id = line.strip() if len(batch_artists) < 50: batch_artists.append(artist_id) else: artists = sp.artists(batch_artists) artists = artists['artists'] for artist in artists: with open(f'artists_dump/{ artist["id"] }.json', 'w') as fout: fout.write(json.dumps(artist)) print(f'[ { index + 1 } / { lines_n } ] { artist["id"] } - { artist["name"] }') batch_artists = [] time.sleep(1) # Print all the IDs of the artists that are missing from the dump. import os with open('./artist_ids.txt', 'r') as artist_ids_file: for artist_id in artist_ids_file: artist_id = artist_id.strip() artist_file_path = f'./artists_dump/{ artist_id }.json' if not os.path.isfile(artist_file_path): print(artist_id) # Generate artists dataset # Read every artist file and generate a CSV file. The category column is not populated yet. import os all_files = [ f for f in os.listdir('./artists_dump/') if os.path.isfile(os.path.join('./artists_dump/', f)) ] all_artists = { 'id': [], 'name': [], 'genres': [], 'category': [], 'followers': [], 'popularity': [] } for current_file in all_files: with open(os.path.join(ARTISTS_DUMP, current_file), 'r') as in_file: artist = json.load(in_file) all_artists['id'].append(artist['id']) all_artists['name'].append( sanitize(artist['name']) ) all_artists['genres'].append( sanitize_list(artist['genres']) ) all_artists['category'].append(None) all_artists['followers'].append(artist['followers']['total']) all_artists['popularity'].append(artist['popularity']) df_artists = pd.DataFrame(all_artists) \ .set_index('id') df_artists.to_csv('artists.csv') # Generate genres dataset # Iterate through the artists dump, collect every genre and save all unique genres in a file. import json import os import numpy all_files = [ f for f in os.listdir('./artists_dump/') if os.path.isfile(os.path.join('./artists_dump/', f)) ] all_genres = [] for current_file in all_files: with open(os.path.join(ARTISTS_DUMP, current_file), 'r') as in_file: data = json.load(in_file) name = data['name'] genres = data['genres'] for genre in genres: all_genres.append(genre) unique_genres = numpy.unique(all_genres) df_genres = pd.DataFrame({ 'genre': unique_genres }) df_genres['genre'] = df_genres['genre'].apply(sanitize) df_genres.set_index('genre', inplace = True) df_genres.to_csv('genres.csv') # Assign categories to every genre df_genres = pd.read_csv('genres.csv') df_genres['category'] = 'other' df_genres.loc[df_genres['genre'].str.contains('disco'), 'category'] = 'disco' df_genres.loc[df_genres['genre'].str.contains('metal'), 'category'] = 'metal' df_genres.loc[df_genres['genre'].str.contains('rock'), 'category'] = 'rock' df_genres.loc[df_genres['genre'].str.contains('classic'), 'category'] = 'classic' df_genres.loc[df_genres['genre'].str.contains('punk'), 'category'] = 'punk' df_genres.loc[df_genres['genre'].str.contains('jazz'), 'category'] = 'jazz' df_genres.loc[df_genres['genre'].str.contains('hip hop'), 'category'] = 'hiphop' df_genres.loc[df_genres['genre'].str.contains('electronic'), 'category'] = 'electronic' df_genres.loc[df_genres['genre'].str.contains('pop'), 'category'] = 'pop' df_genres.loc[df_genres['genre'].str.contains('k-pop'), 'category'] = 'k-pop' df_genres.loc[df_genres['genre'].str.contains('folk'), 'category'] = 'folk' df_genres.loc[df_genres['genre'].str.contains('rap'), 'category'] = 'rap' df_genres.loc[df_genres['genre'].str.contains('trap'), 'category'] = 'trap' df_genres.loc[df_genres['genre'].str.contains('blues'), 'category'] = 'blues' df_genres.loc[df_genres['genre'].str.contains('reggae'), 'category'] = 'reggae' df_genres.loc[df_genres['genre'].str.contains('reggaeton'), 'category'] = 'reggaeton' df_genres = df_genres.set_index('genre') df_genres.to_csv('genres.csv') # Assign categories to every artist df_artists = pd.read_csv('artists.csv', index_col = 'id') df_genres = pd.read_csv('genres.csv', index_col = 'genre') df_genres['name'] = df_genres.index artists_categories = [] for index, artist in df_artists.iterrows(): if artist['genres'] == '[]': artists_categories.append([]) continue artist_genres = artist['genres'][2:-2].split("', '") artist_categories = [] for artist_genre in artist_genres: # Lookup category for artist_genre artist_genre_category = df_genres.loc[artist_genre]['category'] artist_categories.append(artist_genre_category) artists_categories.append(str(list(np.unique(artist_categories)))) df_artists['category'] = artists_categories df_artists.to_csv('artists.csv') # Assign genres, categories and artist's popularity to every song (very slow) df_songs = pd.read_csv('./data/tracks_features.csv', index_col = 'id') df_artists = pd.read_csv('artists.csv', index_col = 'id') # Remove duplicated artists df_artists = df_artists[~df_artists.index.duplicated(keep = 'first')] song_genres = [] song_categories = [] song_popularities = [] for index, song in df_songs.iterrows(): artist_ids = song['artist_ids'][2:-2].split("', '") current_song_genres = [] current_song_categories = [] current_song_popularity = 0 for artist_id in artist_ids: if not artist_id in df_artists.index: continue artist = df_artists.loc[artist_id] artist_genres = artist['genres'][2:-2].split("', '") artist_categories = artist['category'][2:-2].split("', '") artist_popularity = artist['popularity'] current_song_genres.extend(artist_genres) current_song_categories.extend(artist_categories) current_song_popularity = artist_popularity current_song_genres = str(list(np.unique(current_song_genres))) current_song_categories = str(list(np.unique(current_song_categories))) song_genres.append(current_song_genres) song_categories.append(current_song_categories) song_popularities.append(current_song_popularity) df_songs['genres'] = song_genres df_songs['categories'] = song_categories df_songs['artist_popularity'] = song_popularities df_songs.to_csv('songs_extended.csv') # Assign the date of the first release to every artist (very slow) df_songs = pd.read_csv('songs_extended.csv', index_col = 'id') df_songs['release_date'] = pd.to_datetime(df_songs['release_date'], format = '%Y-%m-%d', errors = 'coerce') df_artists = pd.read_csv('artists.csv', index_col = 'id') df_artists.reset_index(inplace = True) first_songs = [] for _, artist in df_artists.iterrows(): artist_id = artist['id'] artist_songs_ = df_songs.loc[df_songs['artist_ids'].str.contains(artist_id)] first_song = None for _, artist_song in artist_songs_.iterrows(): song_release = artist_song['release_date'] if first_song == None or song_release < first_song: first_song = song_release first_songs.append(first_song) df_artists['first_song'] = first_songs df_artists.set_index('id', inplace = True) df_artists.to_csv('artists.csv')